import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class DownloadRecursive2 {
	private String domain;
	private ArrayList<String> urldone = new ArrayList<String>();

	/**
	 * 
	 * @param line a line from the html page
	 * @return ArrayList<String> with all URLS in that line
	 */
	public ArrayList<String> getURLSfromString(String line) {
		if (line == null)
			return new ArrayList<String>();
		// if (!line.contains("href"))
		// return new ArrayList<String>();

		ArrayList<String> urls = new ArrayList<String>();

		Pattern p2 = Pattern.compile("[sS][rR][cC]=\"[^\"]*\"");
		Matcher m2 = p2.matcher(line);
		while (m2.find()) {
			String u = m2.group();
			
			if (!u.contains("?") && !u.contains("#")) {
				System.out.println("gevonden : " + u);
				u = u.replaceAll("[sS][rR][cC]=\"", "");
				u = u.replace("\"", "");
				urls.add(u);
			}
		}

		Pattern p = Pattern.compile("[hH][rR][eE][fF]=\"[^\"]*\"");
		Matcher m = p.matcher(line);
		while (m.find()) {
			String u = m.group();
			
			if (!u.contains("?") && !u.contains("#") ) {
				System.out.println("gevonden : " + u);
				u = u.replaceAll("[hH][rR][eE][fF]=\"", "");
				u = u.replace("\"", "");
				urls.add(u);
			}
		}
		
		Pattern p3 = Pattern.compile("[hH][rR][eE][fF]='[^']*'");
		Matcher m3 = p3.matcher(line);
		while (m3.find()) {
			String u = m3.group();
			
			if (!u.contains("?") && !u.contains("#") ) {
				System.out.println("gevonden test: " + u);
				u = u.replaceAll("[hH][rR][eE][fF]='", "");
				u = u.replace("'", "");
				urls.add(u);
			}
		}
		
		Pattern p4 = Pattern.compile("[sS][rR][cC]='[^']*'");
		Matcher m4 = p4.matcher(line);
		while (m4.find()) {
			String u = m4.group();
			
			if (!u.contains("?") && !u.contains("#")) {
				System.out.println("gevonden : " + u);
				u = u.replaceAll("[sS][rR][cC]='", "");
				u = u.replace("'", "");
				urls.add(u);
			}
		}



		ArrayList<String> addUrls = new ArrayList<String>();
		ArrayList<String> remUrls = new ArrayList<String>();
		for (String url : urls) {
			if ((url.startsWith("http") && !url.contains(domain)) || url.contains("#")) {
				remUrls.add(url);
			}
			
			String temp = url;
			if(temp.contains(domain)){
				temp = temp.replace(domain, "");
			}
			if(!temp.contains(".")){
				remUrls.add(url);
			}

			if (!url.startsWith("http")) {
				if (url.startsWith("/"))
					addUrls.add(domain + url);
				else
					addUrls.add(domain + "/" + url);
				remUrls.add(url);
			}
		
		}
		urls.addAll(addUrls);
		urls.removeAll(remUrls);

		return urls;
	}

	public ArrayList<String> getURLS(ArrayList<String> html) {
		ArrayList<String> urls = new ArrayList<String>();
		for (String line : html) {
			urls.addAll(getURLSfromString(line));
		}
		return urls;
	}

	public void downloadWebsite(URL url) throws IOException {
		domain = url.toString();
		System.out.println("domain = " + domain);
		downloadRecursive(url, 0);
	}

	public void downloadRecursive(URL url, int depth) throws IOException {
		if (depth > 10)
			return;
		if (urldone.contains(url.toString()))
			return;
		urldone.add(url.toString());

		DownloadURL rcDownload = new DownloadURL();
		ArrayList<String> html = rcDownload.downloadURL(url, domain);
		if (html.isEmpty())
			return;
		for (int i = 0; i < depth; i++)
			System.out.print(" ");
		System.out.println("bezig met url " + url);
		ArrayList<String> urls = getURLS(html);
		// ArrayList<String> urls = new ArrayList<String>();

		for (String urlb : urls) {
			for (int i = 0; i < depth; i++)
				System.out.print(" ");
			System.out.println("-->" + urlb);
			downloadRecursive(new URL(urlb), depth + 1);
		}
	}

}
